'''
Created on Feb 23, 2011

@author: mikeg

Python script, capable of determining which BLOb files should be imported based on a filter time window given as
command line args.  

Tested on a folder of SNMP and NMS BLObs:
$ du -h rrt2_hydra_all_blobs_pre_fix
1.3G

Time elapsed = 169.282504082 seconds
'''

"""TODO: 
    1. Test portability on Windoze
    2. Look at request time in blob cut header
    3. Test copy methods that preserve file stats
"""
from optparse import OptionParser
import os
import re
import datetime
import time
import shutil

def parse_cmd_line():
    """Gather command line arguments and handle them accordingly."""
    
    parse_opts = OptionParser()
    parse_opts.add_option("-s", "--start", dest="start_time", action="store",
                          metavar="START_TIME", help="Use -s to specify the start time. \
                          START_TIME format is YYYY-MM-DD-HH-MM-SS.")
    
    parse_opts.add_option("-e", "--end", dest="end_time", action="store",
                          metavar="END_TIME", help="Use -e to specify the end time. \
                          END_TIME format is YYYY-MM-DD-HH-MM-SS.")
    
    parse_opts.add_option("-i", "--input", dest="input_dir", action="store",
                          metavar="INPUT_DIR", help="Use -i to specify the input file dir.")
    
    parse_opts.add_option("-o", "--output", dest="output_dir", action="store",
                          metavar="OUTPUT_DIR", help="Use -o to specify the output file dir.")
    
    parse_opts.add_option("-x", "--ext", dest="file_ext", action="store",
                          metavar="FILE_EXT", help="Use -x to specify the input BLOb file extension. \
                          The default extension is [ .msb ].  This is used when searching through the input directory for BLObs.")
    (options, args) = parse_opts.parse_args()
    
    if options.start_time is None:
        print "No start time was given.  Please supply a start time using the -s flag."
        return None
    
    if options.end_time is None:
        print "No end time was given.  Please supply an end time using the -e flag."
        return None
    
    if options.end_time < options.start_time:
        print "The end time supplied exists further in the past than the start time supplied.  This cannot be."
        return None
    
    if options.input_dir is None:
        print "No input directory was given.  Please supply an input directory using the -i flag."
        return None
    
    if options.output_dir is None:
        print "No output directory was given.  Please supply an output directory using the -o flag."
        return None
    
    if options.file_ext is None:
        options.file_ext = ".msb"
        
    print "Using input parameters:"
    print "Start Time: ", options.start_time
    print "End Time: ", options.end_time
    print "Input Dir: ", options.input_dir
    print "Output Dir: ", options.output_dir
    
    return options

def search_input_dir(input_dir, file_ext):
    """Search an input directory for files with .msb extention and add them to a list. Return the list."""
    file_list = []
    num_files = 0
    for root, subFolders, files in os.walk(input_dir):
        for file in files:
            num_files += 1
            if file[-4:len(file)] == file_ext:
            #print file[-3:len(file)]
                file_list.append(os.path.join(root, file))
    
    return file_list, num_files

def search_times(file):
    """Search the input file for timestamps and add them to a list.  Returns the list."""
    blob_f = open(file, "rb")
    buf = blob_f.read()
    timestamp_regex = re.compile("\d{8}\.\d{6}.\d{6}")

    match_list = re.findall(timestamp_regex, buf)
    blob_f.close()
    match_list_len = len(match_list)
    match_list_short = []
    if match_list_len > 0:
        match_list_short = [match_list[0], match_list[match_list_len-1]]
    return match_list_short

def format_timestamps(time_list):
    """Formats the timestamps into UTC time so that they can be compared easily."""
    time_year = ""
    time_month = ""
    time_day = ""
    time_hours = ""
    time_minutes = ""
    time_seconds = ""
    time_usec = ""
    time_list_epoch = []
    for time_str in time_list:
        time_year = int(time_str[0:4])
        time_month = int(time_str[4:6])
        time_day = int(time_str[6:8])
        time_hours = int(time_str[9:11])
        time_minutes = int(time_str[11:13])
        time_seconds = int(time_str[13:15])
        time_usec = int(time_str[16:len(time_str)]) 
        time_datetime = datetime.datetime(time_year, time_month, time_day, time_hours, time_minutes, time_seconds, time_usec)
        time_epoch = time.mktime(time_datetime.timetuple()) - time.timezone
        time_list_epoch.append(time_epoch)
    return time_list_epoch

def format_input_times(start_time, end_time):
    """Formats the times given as args into epoch times that can be compared easily."""
    
    time_regex = re.compile("\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}")
    if not time_regex.match(start_time):
        print "Please format the start time properly. Use -s flag and format it like this: YYYY-MM-DD-HH-MM-SS."
        return None
    if not time_regex.match(end_time):
        print "Please format the end time properly. Use -e flag and format it like this: YYYY-MM-DD-HH-MM-SS."
        return None
    s_time_part = start_time.split("-")
    s_year = int(s_time_part[0])
    s_month = int(s_time_part[1])
    s_day = int(s_time_part[2])
    s_hour = int(s_time_part[3])
    s_min = int(s_time_part[4])
    s_sec = int(s_time_part[5])
    
    e_time_part = end_time.split("-")
    e_year = int(e_time_part[0])
    e_month = int(e_time_part[1])
    e_day = int(e_time_part[2])
    e_hour = int(e_time_part[3])
    e_min = int(e_time_part[4])
    e_sec = int(e_time_part[5])
        
    s_time_dt = datetime.datetime(s_year, s_month, s_day, s_hour, s_min, s_sec)
    e_time_dt = datetime.datetime(e_year, e_month, e_day, e_hour, e_min, e_sec)
    
    s_time_epoch = time.mktime(s_time_dt.timetuple()) - time.timezone
    e_time_epoch = time.mktime(e_time_dt.timetuple()) - time.timezone
    
    return s_time_epoch, e_time_epoch

def compare_time_filter(time_list, start_time, end_time):
    """Compares each timestamp in the list to the start and end times given as command line args.  If any of the times fall \
    within the window, return True, else return False"""
    for a_time in time_list:
        if start_time < a_time and a_time < end_time:
            return True
        else:
            pass
        
    return False
    
def copy_files(file_list, output_dir):
    """This function copies each file in file_list to the output directory, output_dir.""" 
    files_copied = 0
    if not os.path.exists(output_dir):
        print "OUTPUT PATH NOT FOUND. MAKING."
        os.makedirs(output_dir)
    else:
        print "OUTPUT PATH FOUND. NOT MAKING."
        
    for a_file in file_list:
        shutil.copy2(a_file, output_dir)
        files_copied += 1
    return files_copied
    
if __name__ == '__main__':
    # Parse cmd line
    options = parse_cmd_line()
    if options is None:
        exit()
    
    #Begin 
    timer_start = time.time()
    import_list = []
    
    #Search input directory for BLOb files
    file_list, num_files = search_input_dir(options.input_dir, options.file_ext)
    for a_file in file_list:
        #For each file, create a list of times by scraping them from the BLOb.
        time_list = search_times(a_file)
        #Format all timestamps to UTC Epoch for comparison
        time_list_epoch = format_timestamps(time_list)
        s_time_epoch, e_time_epoch = format_input_times(options.start_time, options.end_time)
        #Determine if any of the BLOb cut times fall within the filter time window. If so, add that file to import_list.
        compare_result = compare_time_filter(time_list_epoch, s_time_epoch, e_time_epoch)
        if compare_result:
            import_list.append(a_file)
            
    #Copy files from import_list to output_dir.       
    files_copied = copy_files(import_list, options.output_dir)
    timer_end = time.time()
    elapsed = timer_end - timer_start
    
    #Print statistics
    print "Time elapsed: ", elapsed
    print "Files searched: ", num_files
    print "Files filtered: ", len(import_list)
    print "Files copied: ", files_copied
    
    if files_copied == len(import_list):
        print "Filter successful. You can find the valid files in ", options.output_dir
    else:
        print "Error: Not all of the valid files were copied to ", options.output_dir, "-- Please check the input files \
        and output directory."
#    for file in file_list:
#        search_times(file)


    